import urllib.request
import urllib.error
import re
import time

#定时爬取间隔
timeInterval=10
headers = ("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4549.400 QQBrowser/9.7.12900.400")
request = urllib.request
request.build_opener().addheaders = [headers]
allResult = []
titleResult = []
def getData():
    #取出所有分类
    #请求每个分类，并取出所有的h2标签
    #从h2标签中取出url
    data = request.urlopen("http://blog.csdn.net").read().decode("utf-8")
    #首页分类的链接
    categoryLinkPat = '<li class=".*?"><a href="(.*?)">.*</a></li>'
    #分类下文章的链接标签
    articlelinkPat = '<h2[\s\S]*?</h2>'
    #文章链接标签下  文章的标题和链接
    detailPat = '<h2[\s\S]*?href="(.*?)" target="_blank">\s*?(.*?)\s*?</a>\s*?</h2>'
    categoryResult = re.compile(categoryLinkPat).findall(data)

#爬取所有文章的标题和url
    for url in categoryResult:
        global allResult
        global titleResult
        newUrl = "http://blog.csdn.net" + url
        #打开每个分类url，从data中截取文章url
        try:
            data = request.urlopen(newUrl).read().decode("utf-8")
            articleResult = re.compile(articlelinkPat).findall(data)
            for link in articleResult:
                result = re.search(detailPat,link)
                print("url :" + result.group(1))
                print("title" + result.group(2).strip())
                titleResult.append(result.group(2).strip())
                allResult.append(result.group(1))
        except urllib.error.URLError as err:
            if(hasattr(err,"code")):
                print("错误码" + str(err.code))
            if(hasattr(err,"reason")):
                print(newUrl)
                print("错误原因"+ str(err.reason))

    allResult = list(set(allResult))

#爬取文章并保存
    for i in range(0,len(allResult)):
        print("link:" + allResult[i])
        title = "C:/Users/admin/Desktop/文档/blog/" + titleResult[i] + ".html"
        try:
            urllib.request.urlretrieve(allResult[i],title)
        except Exception as err:
            if (hasattr(err, "code")):
                print("错误码" + str(err.code))
            if (hasattr(err, "reason")):
                print(allResult[i])
                print("错误原因" + str(err.reason))

while True:
    getData()
    time.sleep(timeInterval);



